package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.Njd1ModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Administrator
 * @description 描述
 * @time 2017/4/12 0012
 */
public class Njd1Downloader extends AbstractCommonDownLoader implements Cloneable{
    private Logger logger = Logger.getLogger(Njd1Downloader.class);
    private String url = "http://www.njd1.com/";
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }

    public List<String> getUrl(){
        List<String> urlList = new ArrayList<String>();
        Document document = null;
        long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("Njd1.URLWaitTime"));
        long remianing = Long.valueOf(PropertiesUtil.getVal("Njd1.URLWaitTime"));
        boolean flag = true;
        while(flag&&remianing>=0) {
            try {
                remianing = future - System.currentTimeMillis();

                document = get(url);

                Elements urlElements = document.select("#hd").select("#colNav");

                Elements hrefElments1 = urlElements.select(".navShopping").select("dd").get(0).select("a");
                for(int i = 0;i<hrefElments1.size();i++){
                    String url = hrefElments1.get(i).attr("href");
                    urlList.add(url);
                    System.out.println(url);
                }

                Elements hrefElments2 = urlElements.select(".navLife").select("dd").get(0).select("a");
                for(int i = 0;i<hrefElments2.size();i++){
                    String url = hrefElments2.get(i).attr("href");
                    urlList.add(url);
                    System.out.println(url);
                }

                Elements hrefElments3 = urlElements.select(".navService").select("dd").get(0).select("a");
                for(int i = 0;i<hrefElments3.size();i++){
                    String url = hrefElments3.get(i).attr("href");
                    urlList.add(url);
                    System.out.println(url);
                }

                flag = false;
            }catch (Exception e){
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }


        return urlList;
    }

    public void generateTask(List<String> urlList){
        try {
            for(int i = 0;i<urlList.size();i++){
                Njd1ModuleTask task = new Njd1ModuleTask(urlList.get(i));
                CommonTaskManager.add("Njd1Module", task);
            }
        }catch (Exception e){
            logger.error(e.getMessage(),e);
            e.printStackTrace();
        }
    }

    public JSONObject crawlItem(String content_url) {
        files=new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();

        int page=1;
        try {
            Document document = null;
            try {
                Page pageContent = null;
                try {
                    pageContent = getHtml(content_url);

                    if (pageContent == null && pageContent.getStatusCode() != 200) {
                        pageContent = getHtml(content_url, true);
                    }
                }catch (Exception e){
                    pageContent = getHtml(content_url, true);
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                }

                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                    try {
                        document = pageContent.getHtml().getDocument();
                        document.select(".aimg_tip").remove();
                        Elements contentElements = document.select("#wp").select("#ct");

                        Elements pageElements = contentElements.select("#pgt").select(".pgt");

                        String regex = "[0-9]{1,}";
                        //System.out.println(pageElements.select(".pg"));
                        if(pageElements != null&&pageElements.size()>0&&pageElements.select(".pg").size()>0){
                            String total = pageElements.select(".pg").select("label").get(0).text();


                            Pattern p = Pattern.compile(regex);
                            Matcher m = p.matcher(total);
                            while (m.find()) {
                                page = Integer.parseInt(m.group());
                            }
                        }

                        Elements contentsElements = contentElements.select("#postlist");

                        String readInfo = contentsElements.select(".comiis_viewtop").select(".pls").select(".hm").text();

                        int read_count = 0;
                        int reply_count = 0;

                        if(readInfo!= null&&!"".equals(readInfo)){
                            String[] readInfos = readInfo.split("\\|");

                            Pattern p = Pattern.compile(regex);
                            Matcher m = p.matcher(readInfos[0]);
                            while (m.find()) {
                                read_count = Integer.parseInt(m.group());
                            }

                            Pattern p1 = Pattern.compile(regex);
                            Matcher m1 = p1.matcher(readInfos[1]);
                            while (m1.find()) {
                                reply_count = Integer.parseInt(m1.group());
                            }

                        }

                        data.put("read_count",read_count);
                        data.put("reply_count",reply_count);
                        data.put("url",content_url);

                        String id = content_url.split("-")[1];

                        data.put("id",id);

                        String title = contentsElements.select(".comiis_viewtop").select("#thread_subject").text();

                        Elements postList = contentsElements.select("div[id~=post_[0-9]{1,}]");

                        String content_id = postList.get(0).attr("id").split("_")[1];

                        Elements userInfo  = postList.get(0).select(".plhin").select(".pls");

                        String publish_user = userInfo.select(".authi").text();

                        String publish_user_id = userInfo.select(".authi").select("a").get(0).attr("href").split("&")[1].split("=")[1];

                        String publish_user_photo = userInfo.select(".avatar").select("a").get(0).select("img").get(0).attr("src");


                        data.put("title",title);
                        data.put("publish_user_id",publish_user_id);
                        data.put("publish_user",publish_user);
                        data.put("publish_user_photo",publish_user_photo);


                        Elements contentInfo = postList.get(0).select(".plhin").select(".plc");

                        String timeInfo = contentInfo.select(".pti").select(".authi").select("#authorposton"+content_id).text();
                        String publish_time = "";
                        String timeRegex = "[0-9]{1,}.*";

                        Pattern p = Pattern.compile(timeRegex);
                        Matcher m = p.matcher(timeInfo);

                        while (m.find()) {
                            publish_time = m.group()+":00";
                        }

                        if("".equals(publish_time)||publish_time==null){
                            System.out.println(timeInfo);
                        }

                        data.put("publish_time",publish_time);


                        String content = contentInfo.select(".pct").select(".pcb").select(".t_fsz").html();

                        data.put("content",FileUtils.filterImg(content));
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements1=contentInfo.select(".pct").select(".pcb").select(".t_fsz").select("img[zoomfile]");//图片
                        Elements imgElements2=contentInfo.select(".pct").select(".pcb").select(".t_fsz").select("img[smilieid]");//表情包
                        if(imgElements1.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements1.size();imgIndex++) {
                                Element imgElement = imgElements1.get(imgIndex);
                                String img_url = imgElement.absUrl("zoomfile");
                                img_urls.add(img_url);
                            }
                        }
                        if(imgElements2.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements2.size();imgIndex++) {
                                Element imgElement = imgElements2.get(imgIndex);
                                String img_url = imgElement.absUrl("src");
                                img_urls.add(img_url);
                            }
                        }

                        data.put("img_urls",img_urls);

                        String spider_time = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);

                        data.put("spider_time",spider_time);

                        long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                        String filePrefix="Njd1";//文件前缀
                        String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
                        String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                        String speechesId="Njd1_speeches_"+publishDate+"_"+fileName;
                        File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                        files.put(speechesId+"_pageNo_1",file);
                        //帖子首页的文件

                        JSONArray reply_nodes = new JSONArray();

                        //查询本页的回复信息
                        if(postList.size()>1&&postList!=null){
                            for(int i = 1;i<postList.size();i++){
                                try {
                                    JSONObject reply_object = new JSONObject();

                                    String content_id1 = postList.get(i).attr("id").split("_")[1];

                                    Elements userInfo1 = postList.get(i).select(".plhin").select(".pls");

                                    String publish_user1 = userInfo1.select(".authi").text();

                                    String publish_user_id1 = userInfo1.select(".authi").select("a").get(0).attr("href").split("&")[1].split("=")[1];

                                    String publish_user_photo1 = "";
                                    try {
                                        publish_user_photo1 = userInfo1.select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                                    }catch (Exception e){
                                        logger.error(e.getMessage(),e);
                                        e.printStackTrace();
                                        continue;
                                    }


                                    reply_object.put("publish_user_id", publish_user_id1);
                                    reply_object.put("publish_user", publish_user1);
                                    reply_object.put("publish_user_photo", publish_user_photo1);


                                    Elements contentInfo1 = postList.get(i).select(".plhin").select(".plc");

                                    String timeInfo1 = contentInfo1.select(".pti").select(".authi").select("#authorposton" + content_id1).text();
                                    String publish_time1 = "";
                                    String timeRegex1 = "[0-9]{1,}.*";

                                    Pattern p1 = Pattern.compile(timeRegex1);
                                    Matcher m1 = p1.matcher(timeInfo1);

                                    while (m1.find()) {
                                        publish_time1 = m1.group() + ":00";
                                    }


                                    reply_object.put("publish_time", publish_time1);


                                    String content1 = contentInfo1.select(".pct").select(".pcb").select(".t_fsz").html();

                                    reply_object.put("content", FileUtils.filterImg(content1));
                                    JSONArray reply_img_urls=new JSONArray();
                                    Elements replyImgElements1=contentInfo.select(".pct").select(".pcb").select(".t_fsz").select("img[zoomfile]");//图片
                                    Elements replyImgElements2=contentInfo.select(".pct").select(".pcb").select(".t_fsz").select("img[smilieid]");//表情包
                                    if(replyImgElements1.size()>0){
                                        for(int imgIndex=0;imgIndex<replyImgElements1.size();imgIndex++) {
                                            Element imgElement = replyImgElements1.get(imgIndex);
                                            String img_url = imgElement.absUrl("zoomfile");
                                            reply_img_urls.add(img_url);
                                        }
                                    }
                                    if(replyImgElements2.size()>0){
                                        for(int imgIndex=0;imgIndex<replyImgElements2.size();imgIndex++) {
                                            Element imgElement = replyImgElements2.get(imgIndex);
                                            String img_url = imgElement.absUrl("src");
                                            reply_img_urls.add(img_url);
                                        }
                                    }
                                    reply_object.put("img_urls",reply_img_urls);

                                    String spider_time1 = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);

                                    reply_object.put("spider_time", spider_time1);
                                    reply_object.put("parent_id", id);
                                    reply_object.put("ancestor_id", id);

                                    String reply_url = "http://www.njd1.com/forum.php?mod=redirect&goto=findpost&pid="+content_id1+"&ptid="+id;

                                    reply_object.put("url",reply_url);

                                    reply_nodes.add(reply_object);
                                }catch (Exception e){
                                    logger.error(e.getMessage(),e);
                                    e.printStackTrace();
                                    continue;
                                }


                            }
                        }

                        JSONArray replys = new JSONArray();
                        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                        if(page>=2){
                            int threadNum = Integer.valueOf(PropertiesUtil.getVal("Njd1.ReplyThreadCount"));
                            ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                            try{
                                List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                for (int i = 0; i < splitPages.size(); i++) {
                                    Njd1Downloader downLoader = this.cloneDownLoader();
                                    Njd1Downloader.ReplyCrawler callable = new Njd1Downloader.ReplyCrawler(downLoader, splitPages.get(i), content_url, id,speechesId);
                                    FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                    futureTasks.add(futureTask);
                                    executorService.submit(futureTask);
                                }

                                for (FutureTask<JSONArray> futureTask : futureTasks) {
                                    JSONArray replyArrays = futureTask.get();
                                    for (int i = 0; i < replyArrays.size(); i++) {
                                        JSONObject object = replyArrays.getJSONObject(i);
                                        replys.add(object);
                                    }
                                }
                                if (replys.size() > 0 && replys != null) {
                                    for (int i = 0; i < replys.size(); i++) {
                                        JSONObject reply_object = replys.getJSONObject(i);
                                        reply_nodes.add(reply_object);
                                    }
                                }
                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                        JSONObject reply_object = reply_nodes.getJSONObject(i);
                                        Map<String, Object> map = reply_object;
                                        list.add(map);
                                    }
                                }
                                Collections.sort(list, new Comparator<Map<String, Object>>() {

                                    public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                        String s1 = o1.get("publish_time").toString();
                                        String s2 = o2.get("publish_time").toString();
                                        int flag = s1.compareTo(s2);
                                        return flag;
                                    }
                                });


                            }catch (Exception e){
                                logger.error(e.getMessage(),e);
                            }finally{
                                executorService.shutdown();
                            }
                        }else{
                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                for (int i = 0; i < reply_nodes.size(); i++) {
                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                    Map<String, Object> map = reply_object;
                                    list.add(map);
                                }
                            }
                        }
                        data.put("reply_nodes", list);


                    }catch (Exception e){
                        logger.error(e.getMessage(),e);
                        e.printStackTrace();
                    }

                }else{

                }


            }catch (Exception e){
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }catch (Exception e){
            logger.error(e.getMessage(),e);
            e.printStackTrace();
        }

        item.put("data",data);
        return item;
    }

    public static void main(String[] args) {
        Njd1Downloader downloader = new Njd1Downloader();
        /*List<String> urlList = downloader.getUrl();
        downloader.generateTask(urlList);*/

        downloader.crawlItem("http://www.njd1.com/thread-386786-1-1.html");

    }

    public Njd1Downloader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        Njd1Downloader loader =new Njd1Downloader();
        loader.cookies= (HashSet<Cookie>) this.cookies.clone();
        loader.headers= (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public class ReplyCrawler implements Callable<JSONArray> {
        private Njd1Downloader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();
            for(int k=0;k<pages.size();k++){
                //boolean flag = true;
                long future= System.currentTimeMillis()+Long.valueOf(PropertiesUtil.getVal("KaiDi.WaitTime"));

                try {

                    String getUrl = url.split("-")[0]+"-"+url.split("-")[1]+"-"+pages.get(k)+"-"+url.split("-")[3];


                    Document document = null;
                    Page pageContent = null;

                    try {
                        pageContent = downLoader.getHtml(getUrl);

                        if (pageContent == null && pageContent.getStatusCode() != 200) {
                            pageContent = downLoader.getHtml(getUrl, true);
                        }
                    }catch (Exception e){
                        pageContent = downLoader.getHtml(getUrl, true);
                        logger.error(e.getMessage(),e);
                        e.printStackTrace();
                    }

                    //pageContent = downLoader.getHtml(getUrl, ProxyPool.proxy());


                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        document = pageContent.getHtml().getDocument();
                        document.select(".aimg_tip").remove();
                        File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                        files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                        Elements contentElements = document.select("#wp").select("#ct");

                        Elements contentsElements = contentElements.select("#postlist");

                        Elements postList = contentsElements.select("div[id~=post_[0-9]{1,}]");
                        for (int i = 0; i < postList.size(); i++) {
                            try {
                                JSONObject reply_object = new JSONObject();

                                String content_id1 = postList.get(i).attr("id").split("_")[1];

                                Elements userInfo1 = postList.get(i).select(".plhin").select(".pls");

                                String publish_user1 = userInfo1.select(".authi").text();

                                String publish_user_id1 = userInfo1.select(".authi").select("a").get(0).attr("href").split("&")[1].split("=")[1];

                                String publish_user_photo1 = userInfo1.select(".avatar").select("a").get(0).select("img").get(0).attr("src");


                                reply_object.put("publish_user_id", publish_user_id1);
                                reply_object.put("publish_user", publish_user1);
                                reply_object.put("publish_user_photo", publish_user_photo1);


                                Elements contentInfo1 = postList.get(i).select(".plhin").select(".plc");

                                String timeInfo1 = contentInfo1.select(".pti").select(".authi").select("#authorposton" + content_id1).text();
                                String publish_time1 = "";
                                String timeRegex1 = "[0-9]{1,}.*";

                                Pattern p1 = Pattern.compile(timeRegex1);
                                Matcher m1 = p1.matcher(timeInfo1);

                                while (m1.find()) {
                                    publish_time1 = m1.group() + ":00";
                                }


                                reply_object.put("publish_time", publish_time1);


                                String content1 = contentInfo1.select(".pct").select(".pcb").select(".t_fsz").html();

                                reply_object.put("content", content1);

                                JSONArray reply_img_urls=new JSONArray();
                                Elements replyImgElements1=contentInfo1.select(".pct").select(".pcb").select(".t_fsz").select("img[zoomfile]");//图片
                                Elements replyImgElements2=contentInfo1.select(".pct").select(".pcb").select(".t_fsz").select("img[smilieid]");//表情包
                                if(replyImgElements1.size()>0){
                                    for(int imgIndex=0;imgIndex<replyImgElements1.size();imgIndex++) {
                                        Element imgElement = replyImgElements1.get(imgIndex);
                                        String img_url = imgElement.absUrl("zoomfile");
                                        reply_img_urls.add(img_url);
                                    }
                                }
                                if(replyImgElements2.size()>0){
                                    for(int imgIndex=0;imgIndex<replyImgElements2.size();imgIndex++) {
                                        Element imgElement = replyImgElements2.get(imgIndex);
                                        String img_url = imgElement.absUrl("src");
                                        reply_img_urls.add(img_url);
                                    }
                                }
                                reply_object.put("img_urls",reply_img_urls);

                                String spider_time1 = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);

                                reply_object.put("spider_time", spider_time1);
                                reply_object.put("parent_id", parent_id);
                                reply_object.put("ancestor_id", parent_id);

                                String reply_url = "http://www.njd1.com/forum.php?mod=redirect&goto=findpost&pid="+content_id1+"&ptid="+parent_id;

                                reply_object.put("url",reply_url);

                                array.add(reply_object);
                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                                continue;
                            }
                        }
                    }else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                    } else {

                    }



                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    continue;
                }
                //}
            }
            return array;
        }

        public ReplyCrawler(Njd1Downloader downLoader, ArrayList<Integer> pages, String url, String parent_id,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url=url;
            this.parent_id = parent_id;
            this.speeches_id=speeches_id;
        }
    }

}
